__author__ = "Chengye zhao (chengyezhao@gmail.com)"

from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.api import urlfetch
from google.appengine.ext import db
from product import Product, getProductListXml, getProductCategory
import time
import logging

from sitelist import SiteList
from parser import ParserList
from product import Product
from search import *

def dofetch(site, i):
	#Look up site infomation from SiteList 
	try:
		siteInfo = SiteList[site]
	except KeyError:
		logging.warning('Info of %s cannot be found', site)
		return 
	url = siteInfo[1]
	
	#Fetch url content
	try:
		result = urlfetch.fetch(url=url, follow_redirects = False)
	except urlfetch.Error:
		if i <= 5:
			time.sleep(10)
			dofetch(site, i+1)
			return
		else:
			logging.error('Site %s url %s cannot connect', site, url)
			return 

	newProduct = ParserList[site](site, result.content)
	oldProduct = db.GqlQuery("SELECT * from Product where site = :1", site)

	#get product to delete and product to insert
	newProductSet = set([p for p in newProduct])
	oldProductSet = set([p for p in oldProduct])

	productInsert = newProductSet - oldProductSet
	productDelete = oldProductSet - newProductSet

	for product in productDelete:
		product.delete()
		#logging.info(product.url + " deleted")

	#Insert new tuangou record  
	url_appear = {}
	for product in productInsert:
		if not url_appear.has_key(product.url):
			if not product.category:
				product.category = getProductCategory(product.title.encode('utf-8'))
			product.put()
			#logging.info(product.url + " inserted")
			url_appear[product.url] = True
	
	#update search index
	updateIndex(productInsert, productDelete)
	return 

class FetchJob(webapp.RequestHandler):
    def get(self):
		dofetch(self.request.get('site'), 1)
		productlist = db.GqlQuery("SELECT * from Product where site = :1", self.request.get('site'))
		self.response.out.write(getProductListXml(productlist))

application = webapp.WSGIApplication(
	[('/fetch', FetchJob)],
	debug=True)

def main():
	run_wsgi_app(application)

if __name__ == "__main__":
	main()
